{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.cross_validation import train_test_split\n",
    "import xgboost as xgb\n",
    "import os\n",
    "import pickle\n",
    "import json\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('language_dict.json','r') as fopen:\n",
    "    languages = json.load(fopen)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### You can get the dataset from [here](https://tatoeba.org/eng/downloads)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>cmn</th>\n",
       "      <th>我們試試看！</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>cmn</td>\n",
       "      <td>我该去睡觉了。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>cmn</td>\n",
       "      <td>你在干什麼啊？</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>cmn</td>\n",
       "      <td>這是什麼啊？</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>cmn</td>\n",
       "      <td>今天是６月１８号，也是Muiriel的生日！</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>cmn</td>\n",
       "      <td>生日快乐，Muiriel！</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   1  cmn                  我們試試看！\n",
       "0  2  cmn                 我该去睡觉了。\n",
       "1  3  cmn                 你在干什麼啊？\n",
       "2  4  cmn                  這是什麼啊？\n",
       "3  5  cmn  今天是６月１８号，也是Muiriel的生日！\n",
       "4  6  cmn           生日快乐，Muiriel！"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lang = pd.read_csv('sentences.csv',sep='\\t')\n",
    "lang = lang.dropna()\n",
    "lang.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_langs = ['zlm','eng','ind']\n",
    "lang.loc[~lang.cmn.isin(selected_langs),'cmn'] = 'OTHER'\n",
    "selected_langs.append('OTHER')\n",
    "sentences, langs = [], []\n",
    "for i in selected_langs:\n",
    "    filtered = lang.loc[lang.cmn == i]\n",
    "    sentences += filtered.iloc[:80000,-1].tolist()\n",
    "    langs += filtered.iloc[:80000,1].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "del lang"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array(['OTHER', 'eng', 'ind', 'zlm'], dtype='<U5'),\n",
       " array([80000, 80000, 11808,    91]))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(langs,return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for file in ['negative','positive']:\n",
    "    with open(file,'r') as fopen:\n",
    "        bm = (' '.join(fopen.read().split('\\n'))).split()\n",
    "        new_langs = [' '.join(bm[i:i+4]) for i in range(0, len(bm), 4)] \n",
    "        sentences += new_langs\n",
    "        langs += ['zlm'] * len(new_langs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def simple_textcleaning_language_detection(string):\n",
    "    string = re.sub('[^A-Za-z ]+', ' ', string)\n",
    "    string = filter(None, string.split())\n",
    "    string = [y.strip() for y in string if len(y) > 1]\n",
    "    return ' '.join(string).lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "bm = ''\n",
    "for i in [i for i in os.listdir(os.getcwd()) if i.find('isu')>=0][:20]:\n",
    "    with open(i,'r') as fopen:\n",
    "        isu = json.load(fopen)\n",
    "    bm += ' '.join([simple_textcleaning_language_detection(i['summary']) for i in isu if i['language']=='id'])\n",
    "bm = bm.split()\n",
    "new_langs = [' '.join(bm[i:i+4]) for i in range(0, len(bm), 4)] \n",
    "sentences += new_langs\n",
    "langs += ['zlm'] * len(new_langs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array(['OTHER', 'eng', 'ind', 'zlm'], dtype='<U5'),\n",
       " array([80000, 80000, 11808, 73106]))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(langs,return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(244914, 783541)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target = LabelEncoder().fit_transform(langs)\n",
    "bow_chars = CountVectorizer(ngram_range=(2, 4), analyzer='char').fit(sentences)\n",
    "vectors = bow_chars.transform(sentences)\n",
    "vectors.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, test_X, train_Y, test_Y = train_test_split(vectors, target, test_size = 0.2)\n",
    "del vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation-mlogloss:1.25802\n",
      "Will train until validation-mlogloss hasn't improved in 100 rounds.\n",
      "[5]\tvalidation-mlogloss:0.797518\n",
      "[10]\tvalidation-mlogloss:0.559224\n",
      "[15]\tvalidation-mlogloss:0.411225\n",
      "[20]\tvalidation-mlogloss:0.314492\n",
      "[25]\tvalidation-mlogloss:0.250854\n",
      "[30]\tvalidation-mlogloss:0.207083\n",
      "[35]\tvalidation-mlogloss:0.175508\n",
      "[40]\tvalidation-mlogloss:0.151243\n",
      "[45]\tvalidation-mlogloss:0.133398\n",
      "[50]\tvalidation-mlogloss:0.119393\n",
      "[55]\tvalidation-mlogloss:0.108089\n",
      "[60]\tvalidation-mlogloss:0.098856\n",
      "[65]\tvalidation-mlogloss:0.090922\n",
      "[70]\tvalidation-mlogloss:0.084587\n",
      "[75]\tvalidation-mlogloss:0.079036\n",
      "[80]\tvalidation-mlogloss:0.073985\n",
      "[85]\tvalidation-mlogloss:0.069712\n",
      "[90]\tvalidation-mlogloss:0.065984\n",
      "[95]\tvalidation-mlogloss:0.062644\n",
      "[100]\tvalidation-mlogloss:0.059571\n",
      "[105]\tvalidation-mlogloss:0.056827\n",
      "[110]\tvalidation-mlogloss:0.054316\n",
      "[115]\tvalidation-mlogloss:0.051973\n",
      "[120]\tvalidation-mlogloss:0.049912\n",
      "[125]\tvalidation-mlogloss:0.04793\n",
      "[130]\tvalidation-mlogloss:0.046164\n",
      "[135]\tvalidation-mlogloss:0.044492\n",
      "[140]\tvalidation-mlogloss:0.042962\n",
      "[145]\tvalidation-mlogloss:0.041602\n",
      "[150]\tvalidation-mlogloss:0.040325\n",
      "[155]\tvalidation-mlogloss:0.039144\n",
      "[160]\tvalidation-mlogloss:0.038029\n",
      "[165]\tvalidation-mlogloss:0.036992\n",
      "[170]\tvalidation-mlogloss:0.035996\n",
      "[175]\tvalidation-mlogloss:0.035034\n",
      "[180]\tvalidation-mlogloss:0.034179\n",
      "[185]\tvalidation-mlogloss:0.033317\n",
      "[190]\tvalidation-mlogloss:0.032585\n",
      "[195]\tvalidation-mlogloss:0.031833\n",
      "[200]\tvalidation-mlogloss:0.031127\n",
      "[205]\tvalidation-mlogloss:0.030467\n",
      "[210]\tvalidation-mlogloss:0.029824\n",
      "[215]\tvalidation-mlogloss:0.029171\n",
      "[220]\tvalidation-mlogloss:0.028597\n",
      "[225]\tvalidation-mlogloss:0.028064\n",
      "[230]\tvalidation-mlogloss:0.027479\n",
      "[235]\tvalidation-mlogloss:0.027004\n",
      "[240]\tvalidation-mlogloss:0.026564\n",
      "[245]\tvalidation-mlogloss:0.026116\n",
      "[250]\tvalidation-mlogloss:0.025672\n",
      "[255]\tvalidation-mlogloss:0.025303\n",
      "[260]\tvalidation-mlogloss:0.024939\n",
      "[265]\tvalidation-mlogloss:0.024545\n",
      "[270]\tvalidation-mlogloss:0.024207\n",
      "[275]\tvalidation-mlogloss:0.023874\n",
      "[280]\tvalidation-mlogloss:0.023579\n",
      "[285]\tvalidation-mlogloss:0.023302\n",
      "[290]\tvalidation-mlogloss:0.02301\n",
      "[295]\tvalidation-mlogloss:0.022743\n",
      "[300]\tvalidation-mlogloss:0.022515\n",
      "[305]\tvalidation-mlogloss:0.022246\n",
      "[310]\tvalidation-mlogloss:0.022014\n",
      "[315]\tvalidation-mlogloss:0.02179\n",
      "[320]\tvalidation-mlogloss:0.021585\n",
      "[325]\tvalidation-mlogloss:0.021343\n",
      "[330]\tvalidation-mlogloss:0.021113\n",
      "[335]\tvalidation-mlogloss:0.020912\n",
      "[340]\tvalidation-mlogloss:0.020737\n",
      "[345]\tvalidation-mlogloss:0.020529\n",
      "[350]\tvalidation-mlogloss:0.02035\n",
      "[355]\tvalidation-mlogloss:0.020184\n",
      "[360]\tvalidation-mlogloss:0.020001\n",
      "[365]\tvalidation-mlogloss:0.019843\n",
      "[370]\tvalidation-mlogloss:0.0197\n",
      "[375]\tvalidation-mlogloss:0.019568\n",
      "[380]\tvalidation-mlogloss:0.019416\n",
      "[385]\tvalidation-mlogloss:0.019287\n",
      "[390]\tvalidation-mlogloss:0.019167\n",
      "[395]\tvalidation-mlogloss:0.019037\n",
      "[400]\tvalidation-mlogloss:0.018919\n",
      "[405]\tvalidation-mlogloss:0.01879\n",
      "[410]\tvalidation-mlogloss:0.018688\n",
      "[415]\tvalidation-mlogloss:0.018592\n",
      "[420]\tvalidation-mlogloss:0.018489\n",
      "[425]\tvalidation-mlogloss:0.018361\n",
      "[430]\tvalidation-mlogloss:0.018263\n",
      "[435]\tvalidation-mlogloss:0.018156\n",
      "[440]\tvalidation-mlogloss:0.018084\n",
      "[445]\tvalidation-mlogloss:0.017984\n",
      "[450]\tvalidation-mlogloss:0.01791\n",
      "[455]\tvalidation-mlogloss:0.017825\n",
      "[460]\tvalidation-mlogloss:0.017742\n",
      "[465]\tvalidation-mlogloss:0.017669\n",
      "[470]\tvalidation-mlogloss:0.017586\n",
      "[475]\tvalidation-mlogloss:0.017503\n",
      "[480]\tvalidation-mlogloss:0.01743\n",
      "[485]\tvalidation-mlogloss:0.017373\n",
      "[490]\tvalidation-mlogloss:0.017313\n",
      "[495]\tvalidation-mlogloss:0.01725\n",
      "[500]\tvalidation-mlogloss:0.017198\n",
      "[505]\tvalidation-mlogloss:0.017116\n",
      "[510]\tvalidation-mlogloss:0.017051\n",
      "[515]\tvalidation-mlogloss:0.016991\n",
      "[520]\tvalidation-mlogloss:0.016944\n",
      "[525]\tvalidation-mlogloss:0.016883\n",
      "[530]\tvalidation-mlogloss:0.016816\n",
      "[535]\tvalidation-mlogloss:0.016767\n",
      "[540]\tvalidation-mlogloss:0.016701\n",
      "[545]\tvalidation-mlogloss:0.016648\n",
      "[550]\tvalidation-mlogloss:0.016604\n",
      "[555]\tvalidation-mlogloss:0.016558\n",
      "[560]\tvalidation-mlogloss:0.016508\n",
      "[565]\tvalidation-mlogloss:0.016461\n",
      "[570]\tvalidation-mlogloss:0.016413\n",
      "[575]\tvalidation-mlogloss:0.016351\n",
      "[580]\tvalidation-mlogloss:0.016326\n",
      "[585]\tvalidation-mlogloss:0.016263\n",
      "[590]\tvalidation-mlogloss:0.016207\n",
      "[595]\tvalidation-mlogloss:0.01617\n",
      "[600]\tvalidation-mlogloss:0.016141\n",
      "[605]\tvalidation-mlogloss:0.016104\n",
      "[610]\tvalidation-mlogloss:0.016085\n",
      "[615]\tvalidation-mlogloss:0.016049\n",
      "[620]\tvalidation-mlogloss:0.016025\n",
      "[625]\tvalidation-mlogloss:0.01599\n",
      "[630]\tvalidation-mlogloss:0.015947\n",
      "[635]\tvalidation-mlogloss:0.015896\n",
      "[640]\tvalidation-mlogloss:0.015891\n",
      "[645]\tvalidation-mlogloss:0.015854\n",
      "[650]\tvalidation-mlogloss:0.015815\n",
      "[655]\tvalidation-mlogloss:0.015787\n",
      "[660]\tvalidation-mlogloss:0.015775\n",
      "[665]\tvalidation-mlogloss:0.015735\n",
      "[670]\tvalidation-mlogloss:0.015714\n",
      "[675]\tvalidation-mlogloss:0.015684\n",
      "[680]\tvalidation-mlogloss:0.015669\n",
      "[685]\tvalidation-mlogloss:0.01565\n",
      "[690]\tvalidation-mlogloss:0.015619\n",
      "[695]\tvalidation-mlogloss:0.01558\n",
      "[700]\tvalidation-mlogloss:0.015565\n",
      "[705]\tvalidation-mlogloss:0.015538\n",
      "[710]\tvalidation-mlogloss:0.015505\n",
      "[715]\tvalidation-mlogloss:0.015496\n",
      "[720]\tvalidation-mlogloss:0.015471\n",
      "[725]\tvalidation-mlogloss:0.015457\n",
      "[730]\tvalidation-mlogloss:0.015424\n",
      "[735]\tvalidation-mlogloss:0.0154\n",
      "[740]\tvalidation-mlogloss:0.01539\n",
      "[745]\tvalidation-mlogloss:0.015363\n",
      "[750]\tvalidation-mlogloss:0.015349\n",
      "[755]\tvalidation-mlogloss:0.015337\n",
      "[760]\tvalidation-mlogloss:0.015322\n",
      "[765]\tvalidation-mlogloss:0.015307\n",
      "[770]\tvalidation-mlogloss:0.015301\n",
      "[775]\tvalidation-mlogloss:0.015293\n",
      "[780]\tvalidation-mlogloss:0.015285\n",
      "[785]\tvalidation-mlogloss:0.015264\n",
      "[790]\tvalidation-mlogloss:0.015244\n",
      "[795]\tvalidation-mlogloss:0.015224\n",
      "[800]\tvalidation-mlogloss:0.015211\n",
      "[805]\tvalidation-mlogloss:0.015192\n",
      "[810]\tvalidation-mlogloss:0.015177\n",
      "[815]\tvalidation-mlogloss:0.015169\n",
      "[820]\tvalidation-mlogloss:0.015156\n",
      "[825]\tvalidation-mlogloss:0.015145\n",
      "[830]\tvalidation-mlogloss:0.015138\n",
      "[835]\tvalidation-mlogloss:0.015126\n",
      "[840]\tvalidation-mlogloss:0.015117\n",
      "[845]\tvalidation-mlogloss:0.015092\n",
      "[850]\tvalidation-mlogloss:0.015077\n",
      "[855]\tvalidation-mlogloss:0.015074\n",
      "[860]\tvalidation-mlogloss:0.015072\n",
      "[865]\tvalidation-mlogloss:0.01506\n",
      "[870]\tvalidation-mlogloss:0.015054\n",
      "[875]\tvalidation-mlogloss:0.01504\n",
      "[880]\tvalidation-mlogloss:0.015031\n",
      "[885]\tvalidation-mlogloss:0.015017\n",
      "[890]\tvalidation-mlogloss:0.015009\n",
      "[895]\tvalidation-mlogloss:0.015002\n",
      "[900]\tvalidation-mlogloss:0.014986\n",
      "[905]\tvalidation-mlogloss:0.014981\n",
      "[910]\tvalidation-mlogloss:0.014977\n",
      "[915]\tvalidation-mlogloss:0.014967\n",
      "[920]\tvalidation-mlogloss:0.014971\n",
      "[925]\tvalidation-mlogloss:0.014967\n",
      "[930]\tvalidation-mlogloss:0.014952\n",
      "[935]\tvalidation-mlogloss:0.014943\n",
      "[940]\tvalidation-mlogloss:0.014933\n",
      "[945]\tvalidation-mlogloss:0.014926\n",
      "[950]\tvalidation-mlogloss:0.014922\n",
      "[955]\tvalidation-mlogloss:0.014901\n",
      "[960]\tvalidation-mlogloss:0.014895\n",
      "[965]\tvalidation-mlogloss:0.014891\n",
      "[970]\tvalidation-mlogloss:0.014884\n",
      "[975]\tvalidation-mlogloss:0.014881\n",
      "[980]\tvalidation-mlogloss:0.014882\n",
      "[985]\tvalidation-mlogloss:0.014876\n",
      "[990]\tvalidation-mlogloss:0.014869\n",
      "[995]\tvalidation-mlogloss:0.014858\n",
      "[1000]\tvalidation-mlogloss:0.014852\n",
      "[1005]\tvalidation-mlogloss:0.014845\n",
      "[1010]\tvalidation-mlogloss:0.014829\n",
      "[1015]\tvalidation-mlogloss:0.014823\n",
      "[1020]\tvalidation-mlogloss:0.014816\n",
      "[1025]\tvalidation-mlogloss:0.014809\n",
      "[1030]\tvalidation-mlogloss:0.014814\n",
      "[1035]\tvalidation-mlogloss:0.014811\n",
      "[1040]\tvalidation-mlogloss:0.014803\n",
      "[1045]\tvalidation-mlogloss:0.014797\n",
      "[1050]\tvalidation-mlogloss:0.014797\n",
      "[1055]\tvalidation-mlogloss:0.014792\n",
      "[1060]\tvalidation-mlogloss:0.014788\n",
      "[1065]\tvalidation-mlogloss:0.014778\n",
      "[1070]\tvalidation-mlogloss:0.01477\n",
      "[1075]\tvalidation-mlogloss:0.014765\n",
      "[1080]\tvalidation-mlogloss:0.014761\n",
      "[1085]\tvalidation-mlogloss:0.014761\n",
      "[1090]\tvalidation-mlogloss:0.014759\n",
      "[1095]\tvalidation-mlogloss:0.014743\n",
      "[1100]\tvalidation-mlogloss:0.014739\n",
      "[1105]\tvalidation-mlogloss:0.014727\n",
      "[1110]\tvalidation-mlogloss:0.01472\n",
      "[1115]\tvalidation-mlogloss:0.014716\n",
      "[1120]\tvalidation-mlogloss:0.014712\n",
      "[1125]\tvalidation-mlogloss:0.014711\n",
      "[1130]\tvalidation-mlogloss:0.014702\n",
      "[1135]\tvalidation-mlogloss:0.014703\n",
      "[1140]\tvalidation-mlogloss:0.014703\n",
      "[1145]\tvalidation-mlogloss:0.01469\n",
      "[1150]\tvalidation-mlogloss:0.014689\n",
      "[1155]\tvalidation-mlogloss:0.014671\n",
      "[1160]\tvalidation-mlogloss:0.014672\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1165]\tvalidation-mlogloss:0.014672\n",
      "[1170]\tvalidation-mlogloss:0.01467\n",
      "[1175]\tvalidation-mlogloss:0.014673\n",
      "[1180]\tvalidation-mlogloss:0.014663\n",
      "[1185]\tvalidation-mlogloss:0.014663\n",
      "[1190]\tvalidation-mlogloss:0.014656\n",
      "[1195]\tvalidation-mlogloss:0.014649\n",
      "[1200]\tvalidation-mlogloss:0.01464\n",
      "[1205]\tvalidation-mlogloss:0.014642\n",
      "[1210]\tvalidation-mlogloss:0.014632\n",
      "[1215]\tvalidation-mlogloss:0.014625\n",
      "[1220]\tvalidation-mlogloss:0.014623\n",
      "[1225]\tvalidation-mlogloss:0.01461\n",
      "[1230]\tvalidation-mlogloss:0.014602\n",
      "[1235]\tvalidation-mlogloss:0.014594\n",
      "[1240]\tvalidation-mlogloss:0.014596\n",
      "[1245]\tvalidation-mlogloss:0.014593\n",
      "[1250]\tvalidation-mlogloss:0.014595\n",
      "[1255]\tvalidation-mlogloss:0.014596\n",
      "[1260]\tvalidation-mlogloss:0.014595\n",
      "[1265]\tvalidation-mlogloss:0.014597\n",
      "[1270]\tvalidation-mlogloss:0.014601\n",
      "[1275]\tvalidation-mlogloss:0.014591\n",
      "[1280]\tvalidation-mlogloss:0.014583\n",
      "[1285]\tvalidation-mlogloss:0.014584\n",
      "[1290]\tvalidation-mlogloss:0.014589\n",
      "[1295]\tvalidation-mlogloss:0.014581\n",
      "[1300]\tvalidation-mlogloss:0.014581\n",
      "[1305]\tvalidation-mlogloss:0.014582\n",
      "[1310]\tvalidation-mlogloss:0.014578\n",
      "[1315]\tvalidation-mlogloss:0.014577\n",
      "[1320]\tvalidation-mlogloss:0.014573\n",
      "[1325]\tvalidation-mlogloss:0.014571\n",
      "[1330]\tvalidation-mlogloss:0.014565\n",
      "[1335]\tvalidation-mlogloss:0.014564\n",
      "[1340]\tvalidation-mlogloss:0.014571\n",
      "[1345]\tvalidation-mlogloss:0.014566\n",
      "[1350]\tvalidation-mlogloss:0.014553\n",
      "[1355]\tvalidation-mlogloss:0.014546\n",
      "[1360]\tvalidation-mlogloss:0.014543\n",
      "[1365]\tvalidation-mlogloss:0.014542\n",
      "[1370]\tvalidation-mlogloss:0.014542\n",
      "[1375]\tvalidation-mlogloss:0.01454\n",
      "[1380]\tvalidation-mlogloss:0.014535\n",
      "[1385]\tvalidation-mlogloss:0.014538\n",
      "[1390]\tvalidation-mlogloss:0.014539\n",
      "[1395]\tvalidation-mlogloss:0.014539\n",
      "[1400]\tvalidation-mlogloss:0.014539\n",
      "[1405]\tvalidation-mlogloss:0.014539\n",
      "[1410]\tvalidation-mlogloss:0.014538\n",
      "[1415]\tvalidation-mlogloss:0.014535\n",
      "[1420]\tvalidation-mlogloss:0.014536\n",
      "[1425]\tvalidation-mlogloss:0.014536\n",
      "[1430]\tvalidation-mlogloss:0.01453\n",
      "[1435]\tvalidation-mlogloss:0.014533\n",
      "[1440]\tvalidation-mlogloss:0.014527\n",
      "[1445]\tvalidation-mlogloss:0.014523\n",
      "[1450]\tvalidation-mlogloss:0.014523\n",
      "[1455]\tvalidation-mlogloss:0.01452\n",
      "[1460]\tvalidation-mlogloss:0.014519\n",
      "[1465]\tvalidation-mlogloss:0.01452\n",
      "[1470]\tvalidation-mlogloss:0.014519\n",
      "[1475]\tvalidation-mlogloss:0.014518\n",
      "[1480]\tvalidation-mlogloss:0.014516\n",
      "[1485]\tvalidation-mlogloss:0.014514\n",
      "[1490]\tvalidation-mlogloss:0.014515\n",
      "[1495]\tvalidation-mlogloss:0.01451\n",
      "[1500]\tvalidation-mlogloss:0.014513\n",
      "[1505]\tvalidation-mlogloss:0.014514\n",
      "[1510]\tvalidation-mlogloss:0.014512\n",
      "[1515]\tvalidation-mlogloss:0.014518\n",
      "[1520]\tvalidation-mlogloss:0.014513\n",
      "[1525]\tvalidation-mlogloss:0.014505\n",
      "[1530]\tvalidation-mlogloss:0.014503\n",
      "[1535]\tvalidation-mlogloss:0.014498\n",
      "[1540]\tvalidation-mlogloss:0.014495\n",
      "[1545]\tvalidation-mlogloss:0.014491\n",
      "[1550]\tvalidation-mlogloss:0.014494\n",
      "[1555]\tvalidation-mlogloss:0.01449\n",
      "[1560]\tvalidation-mlogloss:0.014492\n",
      "[1565]\tvalidation-mlogloss:0.014492\n",
      "[1570]\tvalidation-mlogloss:0.014485\n",
      "[1575]\tvalidation-mlogloss:0.014482\n",
      "[1580]\tvalidation-mlogloss:0.01448\n",
      "[1585]\tvalidation-mlogloss:0.014481\n",
      "[1590]\tvalidation-mlogloss:0.014478\n",
      "[1595]\tvalidation-mlogloss:0.014476\n",
      "[1600]\tvalidation-mlogloss:0.014468\n",
      "[1605]\tvalidation-mlogloss:0.01447\n",
      "[1610]\tvalidation-mlogloss:0.01447\n",
      "[1615]\tvalidation-mlogloss:0.014469\n",
      "[1620]\tvalidation-mlogloss:0.014472\n",
      "[1625]\tvalidation-mlogloss:0.014468\n",
      "[1630]\tvalidation-mlogloss:0.014467\n",
      "[1635]\tvalidation-mlogloss:0.014463\n",
      "[1640]\tvalidation-mlogloss:0.01446\n",
      "[1645]\tvalidation-mlogloss:0.014459\n",
      "[1650]\tvalidation-mlogloss:0.014459\n",
      "[1655]\tvalidation-mlogloss:0.014461\n",
      "[1660]\tvalidation-mlogloss:0.014454\n",
      "[1665]\tvalidation-mlogloss:0.014454\n",
      "[1670]\tvalidation-mlogloss:0.014451\n",
      "[1675]\tvalidation-mlogloss:0.014442\n",
      "[1680]\tvalidation-mlogloss:0.014441\n",
      "[1685]\tvalidation-mlogloss:0.014439\n",
      "[1690]\tvalidation-mlogloss:0.014436\n",
      "[1695]\tvalidation-mlogloss:0.014433\n",
      "[1700]\tvalidation-mlogloss:0.014432\n",
      "[1705]\tvalidation-mlogloss:0.01443\n",
      "[1710]\tvalidation-mlogloss:0.014426\n",
      "[1715]\tvalidation-mlogloss:0.014416\n",
      "[1720]\tvalidation-mlogloss:0.014408\n",
      "[1725]\tvalidation-mlogloss:0.01441\n",
      "[1730]\tvalidation-mlogloss:0.014412\n",
      "[1735]\tvalidation-mlogloss:0.01441\n",
      "[1740]\tvalidation-mlogloss:0.014409\n",
      "[1745]\tvalidation-mlogloss:0.014409\n",
      "[1750]\tvalidation-mlogloss:0.014405\n",
      "[1755]\tvalidation-mlogloss:0.014407\n",
      "[1760]\tvalidation-mlogloss:0.014403\n",
      "[1765]\tvalidation-mlogloss:0.014405\n",
      "[1770]\tvalidation-mlogloss:0.014399\n",
      "[1775]\tvalidation-mlogloss:0.014401\n",
      "[1780]\tvalidation-mlogloss:0.0144\n",
      "[1785]\tvalidation-mlogloss:0.014398\n",
      "[1790]\tvalidation-mlogloss:0.01439\n",
      "[1795]\tvalidation-mlogloss:0.014391\n",
      "[1800]\tvalidation-mlogloss:0.014387\n",
      "[1805]\tvalidation-mlogloss:0.014387\n",
      "[1810]\tvalidation-mlogloss:0.014385\n",
      "[1815]\tvalidation-mlogloss:0.014384\n",
      "[1820]\tvalidation-mlogloss:0.014381\n",
      "[1825]\tvalidation-mlogloss:0.014381\n",
      "[1830]\tvalidation-mlogloss:0.014381\n",
      "[1835]\tvalidation-mlogloss:0.014381\n",
      "[1840]\tvalidation-mlogloss:0.014384\n",
      "[1845]\tvalidation-mlogloss:0.014386\n",
      "[1850]\tvalidation-mlogloss:0.014381\n",
      "[1855]\tvalidation-mlogloss:0.014382\n",
      "[1860]\tvalidation-mlogloss:0.014381\n",
      "[1865]\tvalidation-mlogloss:0.014385\n",
      "[1870]\tvalidation-mlogloss:0.014385\n",
      "[1875]\tvalidation-mlogloss:0.014382\n",
      "[1880]\tvalidation-mlogloss:0.014381\n",
      "[1885]\tvalidation-mlogloss:0.014383\n",
      "[1890]\tvalidation-mlogloss:0.01438\n",
      "[1895]\tvalidation-mlogloss:0.014378\n",
      "[1900]\tvalidation-mlogloss:0.014375\n",
      "[1905]\tvalidation-mlogloss:0.014379\n",
      "[1910]\tvalidation-mlogloss:0.014379\n",
      "[1915]\tvalidation-mlogloss:0.014377\n",
      "[1920]\tvalidation-mlogloss:0.014373\n",
      "[1925]\tvalidation-mlogloss:0.014373\n",
      "[1930]\tvalidation-mlogloss:0.014376\n",
      "[1935]\tvalidation-mlogloss:0.014372\n",
      "[1940]\tvalidation-mlogloss:0.014369\n",
      "[1945]\tvalidation-mlogloss:0.014361\n",
      "[1950]\tvalidation-mlogloss:0.014363\n",
      "[1955]\tvalidation-mlogloss:0.014366\n",
      "[1960]\tvalidation-mlogloss:0.014361\n",
      "[1965]\tvalidation-mlogloss:0.014364\n",
      "[1970]\tvalidation-mlogloss:0.014366\n",
      "[1975]\tvalidation-mlogloss:0.014362\n",
      "[1980]\tvalidation-mlogloss:0.014365\n",
      "[1985]\tvalidation-mlogloss:0.014361\n",
      "[1990]\tvalidation-mlogloss:0.014366\n",
      "[1995]\tvalidation-mlogloss:0.014359\n",
      "[2000]\tvalidation-mlogloss:0.014356\n",
      "[2005]\tvalidation-mlogloss:0.014354\n",
      "[2010]\tvalidation-mlogloss:0.014353\n",
      "[2015]\tvalidation-mlogloss:0.014353\n",
      "[2020]\tvalidation-mlogloss:0.01435\n",
      "[2025]\tvalidation-mlogloss:0.014351\n",
      "[2030]\tvalidation-mlogloss:0.014346\n",
      "[2035]\tvalidation-mlogloss:0.014343\n",
      "[2040]\tvalidation-mlogloss:0.014342\n",
      "[2045]\tvalidation-mlogloss:0.014339\n",
      "[2050]\tvalidation-mlogloss:0.014334\n",
      "[2055]\tvalidation-mlogloss:0.014328\n",
      "[2060]\tvalidation-mlogloss:0.014326\n",
      "[2065]\tvalidation-mlogloss:0.01433\n",
      "[2070]\tvalidation-mlogloss:0.014325\n",
      "[2075]\tvalidation-mlogloss:0.014324\n",
      "[2080]\tvalidation-mlogloss:0.014321\n",
      "[2085]\tvalidation-mlogloss:0.014321\n",
      "[2090]\tvalidation-mlogloss:0.01432\n",
      "[2095]\tvalidation-mlogloss:0.014316\n",
      "[2100]\tvalidation-mlogloss:0.014313\n",
      "[2105]\tvalidation-mlogloss:0.014315\n",
      "[2110]\tvalidation-mlogloss:0.014317\n",
      "[2115]\tvalidation-mlogloss:0.014313\n",
      "[2120]\tvalidation-mlogloss:0.01431\n",
      "[2125]\tvalidation-mlogloss:0.01431\n",
      "[2130]\tvalidation-mlogloss:0.014308\n",
      "[2135]\tvalidation-mlogloss:0.014309\n",
      "[2140]\tvalidation-mlogloss:0.014313\n",
      "[2145]\tvalidation-mlogloss:0.014307\n",
      "[2150]\tvalidation-mlogloss:0.014305\n",
      "[2155]\tvalidation-mlogloss:0.014309\n",
      "[2160]\tvalidation-mlogloss:0.014309\n",
      "[2165]\tvalidation-mlogloss:0.014308\n",
      "[2170]\tvalidation-mlogloss:0.014307\n",
      "[2175]\tvalidation-mlogloss:0.014304\n",
      "[2180]\tvalidation-mlogloss:0.014303\n",
      "[2185]\tvalidation-mlogloss:0.014309\n",
      "[2190]\tvalidation-mlogloss:0.014308\n",
      "[2195]\tvalidation-mlogloss:0.014309\n",
      "[2200]\tvalidation-mlogloss:0.014312\n",
      "[2205]\tvalidation-mlogloss:0.014307\n",
      "[2210]\tvalidation-mlogloss:0.014307\n",
      "[2215]\tvalidation-mlogloss:0.014308\n",
      "[2220]\tvalidation-mlogloss:0.014311\n",
      "[2225]\tvalidation-mlogloss:0.01431\n",
      "[2230]\tvalidation-mlogloss:0.014313\n",
      "[2235]\tvalidation-mlogloss:0.014311\n",
      "[2240]\tvalidation-mlogloss:0.014307\n",
      "[2245]\tvalidation-mlogloss:0.014307\n",
      "[2250]\tvalidation-mlogloss:0.014306\n",
      "[2255]\tvalidation-mlogloss:0.014308\n",
      "[2260]\tvalidation-mlogloss:0.014304\n",
      "[2265]\tvalidation-mlogloss:0.014305\n",
      "[2270]\tvalidation-mlogloss:0.014307\n",
      "[2275]\tvalidation-mlogloss:0.014306\n",
      "[2280]\tvalidation-mlogloss:0.014304\n",
      "Stopping. Best iteration:\n",
      "[2180]\tvalidation-mlogloss:0.014303\n",
      "\n"
     ]
    }
   ],
   "source": [
    "train_d = xgb.DMatrix(train_X, train_Y)\n",
    "test_d = xgb.DMatrix(test_X, test_Y)\n",
    "params_xgd = {\n",
    "    'min_child_weight': 10.0,\n",
    "    'max_depth': 7,\n",
    "    'objective': 'multi:softprob',\n",
    "    'max_delta_step': 1.8,\n",
    "    'num_class': 4,\n",
    "    'colsample_bytree': 0.4,\n",
    "    'subsample': 0.8,\n",
    "    'learning_rate': 0.1,\n",
    "    'gamma': 0.65,\n",
    "    'silent':True,\n",
    "    'eval_metric': 'mlogloss'\n",
    "}\n",
    "model = xgb.train(params_xgd, train_d, 10000, evals=[(test_d, 'validation')], \n",
    "                  early_stopping_rounds=100, verbose_eval=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "      OTHER       1.00      1.00      1.00     16045\n",
      "        eng       1.00      1.00      1.00     16101\n",
      "        ind       0.99      0.96      0.98      2378\n",
      "        zlm       0.99      1.00      1.00     14459\n",
      "\n",
      "avg / total       1.00      1.00      1.00     48983\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predicted = np.argmax(model.predict(xgb.DMatrix(test_X),ntree_limit=model.best_ntree_limit),axis=1)\n",
    "print(metrics.classification_report(test_Y, predicted, target_names = ['OTHER', 'eng', 'ind', 'zlm']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 224 ms, sys: 0 ns, total: 224 ms\n",
      "Wall time: 212 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[9.99844074e-01, 5.22631963e-05, 8.06017197e-05, 2.30375899e-05],\n",
       "       [1.56041992e-07, 9.99998927e-01, 5.57112294e-08, 8.07711103e-07],\n",
       "       [4.69850347e-05, 1.31818715e-05, 9.95305300e-01, 4.63460851e-03],\n",
       "       [9.99860406e-01, 5.04074887e-05, 7.77397945e-05, 1.13682099e-05],\n",
       "       [2.00512186e-05, 9.99937177e-01, 1.08563409e-05, 3.18840721e-05],\n",
       "       [4.35217735e-05, 9.99954700e-01, 2.60621089e-08, 1.70737201e-06],\n",
       "       [2.88806890e-09, 1.00000000e+00, 6.19760021e-10, 1.40123824e-08],\n",
       "       [4.86070462e-07, 6.17368460e-06, 8.47673917e-04, 9.99145627e-01],\n",
       "       [4.92698973e-07, 9.99999404e-01, 7.50260121e-09, 7.90154004e-08],\n",
       "       [3.07888217e-06, 9.99996901e-01, 2.61773336e-09, 7.28362437e-08]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "model.predict(xgb.DMatrix(test_X[:10]),ntree_limit=model.best_ntree_limit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('xgboost-language-detection.pkl','wb') as fopen:\n",
    "    pickle.dump(model,fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('bow-xgboost-language-detection.pkl','wb') as fopen:\n",
    "    pickle.dump(bow_chars,fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
